package crawler;

import java.sql.Connection;
import java.util.ArrayList;
import java.util.HashMap;

import code.extract.BasePageDealer;

import dao.BaseDao;
import dao.PageDao;
import dao.entity.PageEntity;

import net.PageFetcher;
import persist.RefPageDB;
/**
 * Thread for Crawling backlink page
 * BackLinkCrawlerThread
 * @author Lijie Wang, wanglj07@sei.pku.edu.cn
 * 2011-4-20
 */
public class BackLinkCrawlerThread implements Runnable {
	public static final int ONLY_BDB_MODEL = 0;
	public static final int BDB_AND_MYSQL_MODEL = 1;
	private RefPageDB refDB;
	private int model = ONLY_BDB_MODEL;
	private boolean isFinished = false;
	private HashMap<String, PageEntity> uncrawledLinkMapPage = null;
	private int id;
	private int finishedJobNumber = 0;
	private Connection conn = null;
	public void setJob(int id, RefPageDB refDB, Connection conn) {
		this.refDB = refDB;
		this.id = id;
		this.conn = conn;
	}
	
	@Override
	public void run() {
		isFinished = false;
		CrawlItem crawled = null;
		CrawlItem next = null;
		while(true){
			try{
				next = CrawlItemPool.getNextCrawlItem(crawled);
				if(next == null){
					System.out.println("[JOB END] Thread "+id+" finished "+finishedJobNumber+" jobs, going to quit now!");
					break;
				}
				finishedJobNumber++;
				String url = next.getURL();
				System.out.println("Thread "+id+" is Crawling "+next.index+"th url:\t"+url);
				String content = PageFetcher.fetchURL(url);
				System.out.println("Thread "+id+" got "+next.index+"th url:\t"+url);
				if(content != null){
					try {
						synchronized (refDB) {	
							refDB.put(url, content);
							//update
							if (model == BDB_AND_MYSQL_MODEL){
								if (uncrawledLinkMapPage.containsKey(url)){
									PageEntity thisPage = uncrawledLinkMapPage.get(url);
									thisPage.setState(PageEntity.CRAWLED_OK);
									ArrayList<String> metaList = BasePageDealer.extractOneTagContent(content, "meta", "name", "description");
									ArrayList<String> titleList = BasePageDealer.extractOneTagContent(content, "title");
							    	String title = "";
							    	if (titleList.size() > 0){
							    		title = titleList.get(0);
							    		title = BasePageDealer.eraseAllTags(title);
							    	}
									String meta_desc = "";
							    	if (metaList.size() > 0){
							    		meta_desc = metaList.get(0);
							    		meta_desc = BasePageDealer.extractAttribute(meta_desc, "content");
							    	}
							    	metaList = BasePageDealer.extractOneTagContent(content, "meta", "name", "keywords");
							    	String meta_keywords = "";
							    	if (metaList.size() > 0){
							    		meta_keywords = metaList.get(0);
							    		meta_keywords = BasePageDealer.extractAttribute(meta_keywords, "content");
							    	}
							    	thisPage.setTitle(title);
							    	thisPage.setMeta_description(meta_desc);
							    	thisPage.setMeta_keywords(meta_keywords);
							    	PageDao.updatePage(conn, thisPage);
								}else{
									System.err.println("-----error! the url is a gost!" + url);
								}
							}
						}
					} catch (Exception e) {
						e.printStackTrace();
						//update
						if (model == BDB_AND_MYSQL_MODEL){
							if (uncrawledLinkMapPage.containsKey(url)){
								PageEntity thisPage = uncrawledLinkMapPage.get(url);
								thisPage.setState(PageEntity.CRAWLED_ERROR);
								PageDao.updatePage(conn, thisPage);
							}
						}
					}
				}else{
					//update
					if (model == BDB_AND_MYSQL_MODEL){
						if (uncrawledLinkMapPage.containsKey(url)){
							PageEntity thisPage = uncrawledLinkMapPage.get(url);
							thisPage.setState(PageEntity.CRAWLED_ERROR);
							PageDao.updatePage(conn, thisPage);
						}
					}
				}
				crawled = next;
			}catch (Exception e) {
				if(next!=null){
					System.out.println("crawl error: " + next.getURL());
				}
			}
		}
		isFinished = true;
	}

	public void setUncrawledLinkMapPage(HashMap<String, PageEntity> uncrawledPageSet) {
		this.uncrawledLinkMapPage = uncrawledPageSet;
	}

	public HashMap<String, PageEntity> getUncrawledLinkMapPage() {
		return uncrawledLinkMapPage;
	}

	public void setModel(int model) {
		this.model = model;
	}

	public int getModel() {
		return model;
	}

	public boolean isFinished() {
		return isFinished;
	}
}
